raven-matrices-symbolic-center-single:
  id: raven-matrices-symbolic-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test. Matrices composed of a single centered object.
  metrics: [accuracy]

raven-matrices-symbolic-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic/center_single.jsonl


raven-matrices-symbolic-distribute-four:
  id: raven-matrices-symbolic-distribute-four.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test. Matrices composed of four object.
  metrics: [accuracy]

raven-matrices-symbolic-distribute-four.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic/distribute_four.jsonl


raven-matrices-symbolic-distribute-nine:
  id: raven-matrices-symbolic-distribute-nine.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test. Matrices composed of nine object.
  metrics: [accuracy]

raven-matrices-symbolic-distribute-nine.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic/distribute_nine.jsonl


raven-matrices-symbolic-in-center-single-out-center-single:
  id: raven-matrices-symbolic-in-center-single-out-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test. Matrices composed of a small object inside a big object.
  metrics: [accuracy]

raven-matrices-symbolic-in-center-single-out-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic/in_center_single_out_center_single.jsonl


raven-matrices-symbolic-in-distribute-four-out-center-single:
  id: raven-matrices-symbolic-in-distribute-four-out-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test. Matrices composed of four small objects inside a big object.
  metrics: [accuracy]

raven-matrices-symbolic-in-distribute-four-out-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic/in_distribute_four_out_center_single.jsonl


raven-matrices-symbolic-left-center-single-right-center-single:
  id: raven-matrices-symbolic-left-center-single-right-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test. Matrices composed of two objects aligned horizontally.
  metrics: [accuracy]

raven-matrices-symbolic-left-center-single-right-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic/left_center_single_right_center_single.jsonl


raven-matrices-symbolic-up-center-single-down-center-single:
  id: raven-matrices-symbolic-up-center-single-down-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test. Matrices composed of two objects aligned vertically.
  metrics: [accuracy]

raven-matrices-symbolic-up-center-single-down-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic/up_center_single_down_center_single.jsonl




raven-matrices-symbolic-open-center-single:
  id: raven-matrices-symbolic-open-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test with no multiple choices provided. Matrices composed of a single centered object.
  metrics: [accuracy]

raven-matrices-symbolic-open-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic-open/center_single.jsonl


raven-matrices-symbolic-open-distribute-four:
  id: raven-matrices-symbolic-open-distribute-four.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test with no multiple choices provided. Matrices composed of four object.
  metrics: [accuracy]

raven-matrices-symbolic-open-distribute-four.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic-open/distribute_four.jsonl


raven-matrices-symbolic-open-distribute-nine:
  id: raven-matrices-symbolic-open-distribute-nine.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test with no multiple choices provided. Matrices composed of nine object.
  metrics: [accuracy]

raven-matrices-symbolic-open-distribute-nine.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic-open/distribute_nine.jsonl


raven-matrices-symbolic-open-in-center-single-out-center-single:
  id: raven-matrices-symbolic-open-in-center-single-out-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test with no multiple choices provided. Matrices composed of a small object inside a big object.
  metrics: [accuracy]

raven-matrices-symbolic-open-in-center-single-out-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic-open/in_center_single_out_center_single.jsonl


raven-matrices-symbolic-open-in-distribute-four-out-center-single:
  id: raven-matrices-symbolic-open-in-distribute-four-out-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test with no multiple choices provided. Matrices composed of four small objects inside a big object.
  metrics: [accuracy]

raven-matrices-symbolic-open-in-distribute-four-out-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic-open/in_distribute_four_out_center_single.jsonl


raven-matrices-symbolic-open-left-center-single-right-center-single:
  id: raven-matrices-symbolic-open-left-center-single-right-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test with no multiple choices provided. Matrices composed of two objects aligned horizontally.
  metrics: [accuracy]

raven-matrices-symbolic-open-left-center-single-right-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic-open/left_center_single_right_center_single.jsonl


raven-matrices-symbolic-open-up-center-single-down-center-single:
  id: raven-matrices-symbolic-open-up-center-single-down-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a symbolic version of the Raven test with no multiple choices provided. Matrices composed of two objects aligned vertically.
  metrics: [accuracy]

raven-matrices-symbolic-open-up-center-single-down-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/symbolic-open/up_center_single_down_center_single.jsonl



raven-matrices-text-center-single:
  id: raven-matrices-text-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test. Matrices composed of a single centered object.
  metrics: [accuracy]

raven-matrices-text-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text/center_single.jsonl


raven-matrices-text-distribute-four:
  id: raven-matrices-text-distribute-four.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test. Matrices composed of four object.
  metrics: [accuracy]

raven-matrices-text-distribute-four.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text/distribute_four.jsonl


raven-matrices-text-distribute-nine:
  id: raven-matrices-text-distribute-nine.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test. Matrices composed of nine object.
  metrics: [accuracy]

raven-matrices-text-distribute-nine.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text/distribute_nine.jsonl


raven-matrices-text-in-center-single-out-center-single:
  id: raven-matrices-text-in-center-single-out-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test. Matrices composed of a small object inside a big object.
  metrics: [accuracy]

raven-matrices-text-in-center-single-out-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text/in_center_single_out_center_single.jsonl


raven-matrices-text-in-distribute-four-out-center-single:
  id: raven-matrices-text-in-distribute-four-out-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test. Matrices composed of four small objects inside a big object.
  metrics: [accuracy]

raven-matrices-text-in-distribute-four-out-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text/in_distribute_four_out_center_single.jsonl


raven-matrices-text-left-center-single-right-center-single:
  id: raven-matrices-text-left-center-single-right-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test. Matrices composed of two objects aligned horizontally.
  metrics: [accuracy]

raven-matrices-text-left-center-single-right-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text/left_center_single_right_center_single.jsonl


raven-matrices-text-up-center-single-down-center-single:
  id: raven-matrices-text-up-center-single-down-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test. Matrices composed of two objects aligned vertically.
  metrics: [accuracy]

raven-matrices-text-up-center-single-down-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text/up_center_single_down_center_single.jsonl




raven-matrices-text-open-center-single:
  id: raven-matrices-text-open-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test with no multiple choices provided. Matrices composed of a single centered object.
  metrics: [accuracy]

raven-matrices-text-open-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text-open/center_single.jsonl


raven-matrices-text-open-distribute-four:
  id: raven-matrices-text-open-distribute-four.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test with no multiple choices provided. Matrices composed of four object.
  metrics: [accuracy]

raven-matrices-text-open-distribute-four.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text-open/distribute_four.jsonl


raven-matrices-text-open-distribute-nine:
  id: raven-matrices-text-open-distribute-nine.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test with no multiple choices provided. Matrices composed of nine object.
  metrics: [accuracy]

raven-matrices-text-open-distribute-nine.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text-open/distribute_nine.jsonl


raven-matrices-text-open-in-center-single-out-center-single:
  id: raven-matrices-text-open-in-center-single-out-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test with no multiple choices provided. Matrices composed of a small object inside a big object.
  metrics: [accuracy]

raven-matrices-text-open-in-center-single-out-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text-open/in_center_single_out_center_single.jsonl


raven-matrices-text-open-in-distribute-four-out-center-single:
  id: raven-matrices-text-open-in-distribute-four-out-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test with no multiple choices provided. Matrices composed of four small objects inside a big object.
  metrics: [accuracy]

raven-matrices-text-open-in-distribute-four-out-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text-open/in_distribute_four_out_center_single.jsonl


raven-matrices-text-open-left-center-single-right-center-single:
  id: raven-matrices-text-open-left-center-single-right-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test with no multiple choices provided. Matrices composed of two objects aligned horizontally.
  metrics: [accuracy]

raven-matrices-text-open-left-center-single-right-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text-open/left_center_single_right_center_single.jsonl


raven-matrices-text-open-up-center-single-down-center-single:
  id: raven-matrices-text-open-up-center-single-down-center-single.dev.v0
  description: Evaluate the abstract reasoning abilities of the model on a text version of the Raven test with no multiple choices provided. Matrices composed of two objects aligned vertically.
  metrics: [accuracy]

raven-matrices-text-open-up-center-single-down-center-single.dev.v0:
  class: evals.elsuite.basic.includes:Includes
  args:
    samples_jsonl: raven-matrices/text-open/up_center_single_down_center_single.jsonl


